{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "0523.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/yananma/5_programs_per_day/blob/master/0523.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "EdrKG6qUeYYl",
        "colab_type": "text"
      },
      "source": [
        "## 5.1 二维卷积层"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4_1i4FMveuYF",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "outputId": "0e337d46-4ea1-4327-c86b-f38c6f44221e"
      },
      "source": [
        "!pip install mxnet d2lzh"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting mxnet\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/92/6c/c6e5562f8face683cec73f5d4d74a58f8572c0595d54f1fed9d923020bbd/mxnet-1.5.1.post0-py2.py3-none-manylinux1_x86_64.whl (25.4MB)\n",
            "\u001b[K     |████████████████████████████████| 25.4MB 37.6MB/s \n",
            "\u001b[?25hCollecting d2lzh\n",
            "  Downloading https://files.pythonhosted.org/packages/21/cd/a5dbf74fce1a2b0a805488065135a753b4419b29cce109dee960824f1468/d2lzh-0.8.11.tar.gz\n",
            "Collecting graphviz<0.9.0,>=0.8.1\n",
            "  Downloading https://files.pythonhosted.org/packages/53/39/4ab213673844e0c004bed8a0781a0721a3f6bb23eb8854ee75c236428892/graphviz-0.8.4-py2.py3-none-any.whl\n",
            "Requirement already satisfied: requests<3,>=2.20.0 in /usr/local/lib/python3.6/dist-packages (from mxnet) (2.21.0)\n",
            "Requirement already satisfied: numpy<2.0.0,>1.16.0 in /usr/local/lib/python3.6/dist-packages (from mxnet) (1.17.3)\n",
            "Requirement already satisfied: matplotlib in /usr/local/lib/python3.6/dist-packages (from d2lzh) (3.1.1)\n",
            "Requirement already satisfied: jupyter in /usr/local/lib/python3.6/dist-packages (from d2lzh) (1.0.0)\n",
            "Requirement already satisfied: urllib3<1.25,>=1.21.1 in /usr/local/lib/python3.6/dist-packages (from requests<3,>=2.20.0->mxnet) (1.24.3)\n",
            "Requirement already satisfied: idna<2.9,>=2.5 in /usr/local/lib/python3.6/dist-packages (from requests<3,>=2.20.0->mxnet) (2.8)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.6/dist-packages (from requests<3,>=2.20.0->mxnet) (2019.9.11)\n",
            "Requirement already satisfied: chardet<3.1.0,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from requests<3,>=2.20.0->mxnet) (3.0.4)\n",
            "Requirement already satisfied: pyparsing!=2.0.4,!=2.1.2,!=2.1.6,>=2.0.1 in /usr/local/lib/python3.6/dist-packages (from matplotlib->d2lzh) (2.4.2)\n",
            "Requirement already satisfied: cycler>=0.10 in /usr/local/lib/python3.6/dist-packages (from matplotlib->d2lzh) (0.10.0)\n",
            "Requirement already satisfied: python-dateutil>=2.1 in /usr/local/lib/python3.6/dist-packages (from matplotlib->d2lzh) (2.6.1)\n",
            "Requirement already satisfied: kiwisolver>=1.0.1 in /usr/local/lib/python3.6/dist-packages (from matplotlib->d2lzh) (1.1.0)\n",
            "Requirement already satisfied: ipykernel in /usr/local/lib/python3.6/dist-packages (from jupyter->d2lzh) (4.6.1)\n",
            "Requirement already satisfied: qtconsole in /usr/local/lib/python3.6/dist-packages (from jupyter->d2lzh) (4.5.5)\n",
            "Requirement already satisfied: ipywidgets in /usr/local/lib/python3.6/dist-packages (from jupyter->d2lzh) (7.5.1)\n",
            "Requirement already satisfied: notebook in /usr/local/lib/python3.6/dist-packages (from jupyter->d2lzh) (5.2.2)\n",
            "Requirement already satisfied: jupyter-console in /usr/local/lib/python3.6/dist-packages (from jupyter->d2lzh) (5.2.0)\n",
            "Requirement already satisfied: nbconvert in /usr/local/lib/python3.6/dist-packages (from jupyter->d2lzh) (5.6.1)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.6/dist-packages (from cycler>=0.10->matplotlib->d2lzh) (1.12.0)\n",
            "Requirement already satisfied: setuptools in /usr/local/lib/python3.6/dist-packages (from kiwisolver>=1.0.1->matplotlib->d2lzh) (41.4.0)\n",
            "Requirement already satisfied: tornado>=4.0 in /usr/local/lib/python3.6/dist-packages (from ipykernel->jupyter->d2lzh) (4.5.3)\n",
            "Requirement already satisfied: ipython>=4.0.0 in /usr/local/lib/python3.6/dist-packages (from ipykernel->jupyter->d2lzh) (5.5.0)\n",
            "Requirement already satisfied: traitlets>=4.1.0 in /usr/local/lib/python3.6/dist-packages (from ipykernel->jupyter->d2lzh) (4.3.3)\n",
            "Requirement already satisfied: jupyter-client in /usr/local/lib/python3.6/dist-packages (from ipykernel->jupyter->d2lzh) (5.3.4)\n",
            "Requirement already satisfied: pygments in /usr/local/lib/python3.6/dist-packages (from qtconsole->jupyter->d2lzh) (2.1.3)\n",
            "Requirement already satisfied: jupyter-core in /usr/local/lib/python3.6/dist-packages (from qtconsole->jupyter->d2lzh) (4.6.1)\n",
            "Requirement already satisfied: ipython-genutils in /usr/local/lib/python3.6/dist-packages (from qtconsole->jupyter->d2lzh) (0.2.0)\n",
            "Requirement already satisfied: nbformat>=4.2.0 in /usr/local/lib/python3.6/dist-packages (from ipywidgets->jupyter->d2lzh) (4.4.0)\n",
            "Requirement already satisfied: widgetsnbextension~=3.5.0 in /usr/local/lib/python3.6/dist-packages (from ipywidgets->jupyter->d2lzh) (3.5.1)\n",
            "Requirement already satisfied: terminado>=0.3.3; sys_platform != \"win32\" in /usr/local/lib/python3.6/dist-packages (from notebook->jupyter->d2lzh) (0.8.2)\n",
            "Requirement already satisfied: jinja2 in /usr/local/lib/python3.6/dist-packages (from notebook->jupyter->d2lzh) (2.10.3)\n",
            "Requirement already satisfied: prompt-toolkit<2.0.0,>=1.0.0 in /usr/local/lib/python3.6/dist-packages (from jupyter-console->jupyter->d2lzh) (1.0.18)\n",
            "Requirement already satisfied: entrypoints>=0.2.2 in /usr/local/lib/python3.6/dist-packages (from nbconvert->jupyter->d2lzh) (0.3)\n",
            "Requirement already satisfied: pandocfilters>=1.4.1 in /usr/local/lib/python3.6/dist-packages (from nbconvert->jupyter->d2lzh) (1.4.2)\n",
            "Requirement already satisfied: testpath in /usr/local/lib/python3.6/dist-packages (from nbconvert->jupyter->d2lzh) (0.4.2)\n",
            "Requirement already satisfied: defusedxml in /usr/local/lib/python3.6/dist-packages (from nbconvert->jupyter->d2lzh) (0.6.0)\n",
            "Requirement already satisfied: mistune<2,>=0.8.1 in /usr/local/lib/python3.6/dist-packages (from nbconvert->jupyter->d2lzh) (0.8.4)\n",
            "Requirement already satisfied: bleach in /usr/local/lib/python3.6/dist-packages (from nbconvert->jupyter->d2lzh) (3.1.0)\n",
            "Requirement already satisfied: pexpect; sys_platform != \"win32\" in /usr/local/lib/python3.6/dist-packages (from ipython>=4.0.0->ipykernel->jupyter->d2lzh) (4.7.0)\n",
            "Requirement already satisfied: pickleshare in /usr/local/lib/python3.6/dist-packages (from ipython>=4.0.0->ipykernel->jupyter->d2lzh) (0.7.5)\n",
            "Requirement already satisfied: decorator in /usr/local/lib/python3.6/dist-packages (from ipython>=4.0.0->ipykernel->jupyter->d2lzh) (4.4.1)\n",
            "Requirement already satisfied: simplegeneric>0.8 in /usr/local/lib/python3.6/dist-packages (from ipython>=4.0.0->ipykernel->jupyter->d2lzh) (0.8.1)\n",
            "Requirement already satisfied: pyzmq>=13 in /usr/local/lib/python3.6/dist-packages (from jupyter-client->ipykernel->jupyter->d2lzh) (17.0.0)\n",
            "Requirement already satisfied: jsonschema!=2.5.0,>=2.4 in /usr/local/lib/python3.6/dist-packages (from nbformat>=4.2.0->ipywidgets->jupyter->d2lzh) (2.6.0)\n",
            "Requirement already satisfied: ptyprocess; os_name != \"nt\" in /usr/local/lib/python3.6/dist-packages (from terminado>=0.3.3; sys_platform != \"win32\"->notebook->jupyter->d2lzh) (0.6.0)\n",
            "Requirement already satisfied: MarkupSafe>=0.23 in /usr/local/lib/python3.6/dist-packages (from jinja2->notebook->jupyter->d2lzh) (1.1.1)\n",
            "Requirement already satisfied: wcwidth in /usr/local/lib/python3.6/dist-packages (from prompt-toolkit<2.0.0,>=1.0.0->jupyter-console->jupyter->d2lzh) (0.1.7)\n",
            "Requirement already satisfied: webencodings in /usr/local/lib/python3.6/dist-packages (from bleach->nbconvert->jupyter->d2lzh) (0.5.1)\n",
            "Building wheels for collected packages: d2lzh\n",
            "  Building wheel for d2lzh (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for d2lzh: filename=d2lzh-0.8.11-cp36-none-any.whl size=10011 sha256=9f3391a858c7520819222c0799788253f7ef2235ebc0cbc1323e06115560e1b3\n",
            "  Stored in directory: /root/.cache/pip/wheels/bb/4a/3e/81075d0b470000f4b5769c936f64b22be31c6bcfa81fd050d6\n",
            "Successfully built d2lzh\n",
            "Installing collected packages: graphviz, mxnet, d2lzh\n",
            "  Found existing installation: graphviz 0.10.1\n",
            "    Uninstalling graphviz-0.10.1:\n",
            "      Successfully uninstalled graphviz-0.10.1\n",
            "Successfully installed d2lzh-0.8.11 graphviz-0.8.4 mxnet-1.5.1.post0\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LkLTkqFqfIIT",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from mxnet import autograd, nd \n",
        "from mxnet.gluon import nn "
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "D4mm_eRXxvWg",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def corr2d(X, K):  \n",
        "    h, w = K.shape\n",
        "    Y = nd.zeros((X.shape[0] - h + 1, X.shape[1] - w + 1))\n",
        "    for i in range(Y.shape[0]):\n",
        "        for j in range(Y.shape[1]):\n",
        "            Y[i, j] = (X[i: i + h, j: j + w] * K).sum()\n",
        "    return Y"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "T6PKeESjIx6A",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 87
        },
        "outputId": "34e22d76-798a-418f-af68-389b91f3ac20"
      },
      "source": [
        "X = nd.array([[0, 1, 2], [3, 4, 5], [6, 7, 8]])\n",
        "K = nd.array([[0, 1], [2, 3]])\n",
        "corr2d(X, K)"
      ],
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "\n",
              "[[19. 25.]\n",
              " [37. 43.]]\n",
              "<NDArray 2x2 @cpu(0)>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 5
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xI3gM-1OJD-5",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "class Conv2D(nn.Block):\n",
        "    def __init__(self, kernel_size, **kwargs):\n",
        "        super(Conv2D, self).__init__(**kwargs)\n",
        "        self.weight = self.params.get('weight', shape=kernel_size)\n",
        "        self.bias = self.params.get('bias', shape=(1,))\n",
        "\n",
        "    def forward(self, x):\n",
        "        return corr2d(x, self.weight.data()) + self.bias.data()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xFhoC8vDLoPX",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 158
        },
        "outputId": "7f690648-aa46-4621-f187-15c926834fd5"
      },
      "source": [
        "X = nd.ones((6, 8))\n",
        "X[:, 2:6] = 0 \n",
        "X"
      ],
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "\n",
              "[[1. 1. 0. 0. 0. 0. 1. 1.]\n",
              " [1. 1. 0. 0. 0. 0. 1. 1.]\n",
              " [1. 1. 0. 0. 0. 0. 1. 1.]\n",
              " [1. 1. 0. 0. 0. 0. 1. 1.]\n",
              " [1. 1. 0. 0. 0. 0. 1. 1.]\n",
              " [1. 1. 0. 0. 0. 0. 1. 1.]]\n",
              "<NDArray 6x8 @cpu(0)>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 7
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_cX8wMhzZ1Ti",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "K = nd.array([[1, -1]])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "OJz0UAK7aCcE",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 158
        },
        "outputId": "208d779c-84b1-447f-e548-f85069a14858"
      },
      "source": [
        "Y = corr2d(X, K)\n",
        "Y"
      ],
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "\n",
              "[[ 0.  1.  0.  0.  0. -1.  0.]\n",
              " [ 0.  1.  0.  0.  0. -1.  0.]\n",
              " [ 0.  1.  0.  0.  0. -1.  0.]\n",
              " [ 0.  1.  0.  0.  0. -1.  0.]\n",
              " [ 0.  1.  0.  0.  0. -1.  0.]\n",
              " [ 0.  1.  0.  0.  0. -1.  0.]]\n",
              "<NDArray 6x7 @cpu(0)>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 9
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "iNnnEIgUaIab",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 105
        },
        "outputId": "39b08de3-9c1e-48e4-c17e-77c28e7ca100"
      },
      "source": [
        "conv2d = nn.Conv2D(1, kernel_size=(1, 2))\n",
        "conv2d.initialize()\n",
        "\n",
        "X = X.reshape((1, 1, 6, 8))\n",
        "Y = Y.reshape((1, 1, 6, 7))\n",
        "\n",
        "for i in range(10):\n",
        "    with autograd.record():\n",
        "        Y_hat = conv2d(X)\n",
        "        l = (Y_hat - Y) ** 2 \n",
        "    l.backward()\n",
        "    conv2d.weight.data()[:] -= 3e-2 * conv2d.weight.grad()\n",
        "    if (i + 1) % 2 == 0:\n",
        "        print('batch %d, loss %.3f' % (i + 1, l.sum().asscalar()))"
      ],
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "batch 2, loss 4.949\n",
            "batch 4, loss 0.831\n",
            "batch 6, loss 0.140\n",
            "batch 8, loss 0.024\n",
            "batch 10, loss 0.004\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "AfWhEzK3bPcN",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 70
        },
        "outputId": "80cd8f40-4f3c-47f6-fd12-89489f0ac8a7"
      },
      "source": [
        "conv2d.weight.data().reshape((1, 2))"
      ],
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "\n",
              "[[ 0.9895    -0.9873705]]\n",
              "<NDArray 1x2 @cpu(0)>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 11
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "1BCyibyubiB_",
        "colab_type": "text"
      },
      "source": [
        "## 5.2 填充和步幅"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lEFA9f5ib3Ar",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "c26ed388-7ade-47fc-a678-125e59f558c1"
      },
      "source": [
        "from mxnet import nd \n",
        "from mxnet.gluon import nn \n",
        "\n",
        "def comp_conv2d(conv2d, X):\n",
        "    conv2d.initialize()\n",
        "    X = X.reshape((1, 1) + X.shape)\n",
        "    Y = conv2d(X)\n",
        "    return Y.reshape(Y.shape[2:])\n",
        "\n",
        "conv2d = nn.Conv2D(1, kernel_size=3, padding=1)\n",
        "X = nd.random.uniform(shape=(8, 8))\n",
        "comp_conv2d(conv2d, X).shape "
      ],
      "execution_count": 13,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(8, 8)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 13
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MggawGFIOOGI",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "6e4dbb14-6648-4f82-e77d-aaf79256f0d9"
      },
      "source": [
        "conv2d = nn.Conv2D(1, kernel_size=(5, 3), padding=(2, 1))\n",
        "comp_conv2d(conv2d, X).shape"
      ],
      "execution_count": 14,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(8, 8)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 14
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "cIpHk5r7Z8bP",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "07dc3381-09eb-40fe-eb6c-d2a0e871ef92"
      },
      "source": [
        "conv2d = nn.Conv2D(1, kernel_size=3, padding=1, strides=2)\n",
        "comp_conv2d(conv2d, X).shape"
      ],
      "execution_count": 15,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(4, 4)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 15
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "aw5PK0DxdZc2",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "88a01c40-aceb-43d5-8bb7-f373234f3ae2"
      },
      "source": [
        "conv2d = nn.Conv2D(1, kernel_size=(3, 5), padding=(0, 1), strides=(3, 4))\n",
        "comp_conv2d(conv2d, X).shape"
      ],
      "execution_count": 16,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(2, 2)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 16
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "0wSzswIQB6Ao",
        "colab_type": "text"
      },
      "source": [
        "## 5.3 多输入通道和多输出通道"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "tAIy3mJjdqWz",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import d2lzh as d2l \n",
        "from mxnet import nd \n",
        "\n",
        "def corr2d_multi_in(X, K):\n",
        "    return nd.add_n(*[d2l.corr2d(x, k) for x, k in zip(X, K)])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "s_Rt-sgGe2Nk",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 87
        },
        "outputId": "763f2ae8-b0bb-437c-d173-bfcad2a11b4a"
      },
      "source": [
        "X = nd.array([[[0, 1, 2], [3, 4, 5], [6, 7, 8]], \n",
        "        [[1, 2, 3], [4, 5, 6], [7, 8, 9]]])\n",
        "K = nd.array([[[0, 1], [2, 3]], [[1, 2], [3, 4]]])\n",
        "\n",
        "corr2d_multi_in(X, K)"
      ],
      "execution_count": 19,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "\n",
              "[[ 56.  72.]\n",
              " [104. 120.]]\n",
              "<NDArray 2x2 @cpu(0)>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 19
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Ukz4kk1lfYsl",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def corr2d_multi_in_out(X, K):\n",
        "    return nd.stack(*[corr2d_multi_in(X, k) for k in K])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "t-iCnFzpm2a8",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "d4488b2b-e153-4011-c9f9-9be43dcf1e1f"
      },
      "source": [
        "K = nd.stack(K, K+1, K+2)\n",
        "K.shape"
      ],
      "execution_count": 22,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(3, 2, 2, 2)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 22
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_-Htr4VdnMzI",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 386
        },
        "outputId": "4d11e229-ed6c-4fce-8582-5de153c60106"
      },
      "source": [
        "K"
      ],
      "execution_count": 23,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "\n",
              "[[[[0. 1.]\n",
              "   [2. 3.]]\n",
              "\n",
              "  [[1. 2.]\n",
              "   [3. 4.]]]\n",
              "\n",
              "\n",
              " [[[1. 2.]\n",
              "   [3. 4.]]\n",
              "\n",
              "  [[2. 3.]\n",
              "   [4. 5.]]]\n",
              "\n",
              "\n",
              " [[[2. 3.]\n",
              "   [4. 5.]]\n",
              "\n",
              "  [[3. 4.]\n",
              "   [5. 6.]]]]\n",
              "<NDArray 3x2x2x2 @cpu(0)>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 23
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hq6lVdMonQ8A",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 193
        },
        "outputId": "d1338b29-a336-4cb2-d980-ca8106f9d8f2"
      },
      "source": [
        "corr2d_multi_in_out(X, K)"
      ],
      "execution_count": 24,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "\n",
              "[[[ 56.  72.]\n",
              "  [104. 120.]]\n",
              "\n",
              " [[ 76. 100.]\n",
              "  [148. 172.]]\n",
              "\n",
              " [[ 96. 128.]\n",
              "  [192. 224.]]]\n",
              "<NDArray 3x2x2 @cpu(0)>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 24
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fwc5WxlInaIm",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def corr2d_multi_in_out_1x1(X, K):\n",
        "    c_i, h, w = X.shape \n",
        "    c_o = K.shape[0]\n",
        "    X = X.reshape((c_i, h * w))\n",
        "    K = K.reshape((c_o, c_i))\n",
        "    Y = nd.dot(K, X)\n",
        "    return Y.reshape((c_o, h, w))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "QYeFWDJroCxn",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "008e81b8-f910-4060-ca90-ac2fc4cf991a"
      },
      "source": [
        "X = nd.random.uniform(shape=(3, 3, 3))\n",
        "K = nd.random.uniform(shape=(2, 3, 1, 1))\n",
        "Y1 = corr2d_multi_in_out_1x1(X, K)\n",
        "Y2 = corr2d_multi_in_out(X, K)\n",
        "\n",
        "(Y1 - Y2).norm().asscalar() < 1e-6"
      ],
      "execution_count": 27,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "True"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 27
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "AlT4v2q4CH2Z",
        "colab_type": "text"
      },
      "source": [
        "## 5.4 池化层"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dRjHX1DOoajC",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from mxnet import nd \n",
        "from mxnet.gluon import nn \n",
        "\n",
        "def pool2d(X, pool_size, mode='max'):\n",
        "    p_h, p_w = pool_size \n",
        "    Y = nd.zeros((X.shape[0] - p_h + 1, X.shape[1] - p_w + 1))\n",
        "    for i in range(Y.shape[1]):\n",
        "        for j in range(Y.shape[1]):\n",
        "            if mode == 'max':\n",
        "                Y[i, j] = X[i: i + p_h, j: j + p_w].max()\n",
        "            elif mode == 'avg':\n",
        "                Y[i, j] = X[i: i + p_h, j: j+ p_w].mean()\n",
        "    return Y"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "l4-rwPD9rLcY",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 87
        },
        "outputId": "608950cc-3506-4b8e-b164-de5b815f2143"
      },
      "source": [
        "X = nd.array([[0, 1, 2], [3, 4, 5], [6, 7, 8]])\n",
        "pool2d(X, (2, 2))"
      ],
      "execution_count": 33,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "\n",
              "[[4. 5.]\n",
              " [7. 8.]]\n",
              "<NDArray 2x2 @cpu(0)>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 33
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0bOj4m7_rV-b",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 87
        },
        "outputId": "6d1840e0-e523-4829-a3e2-9d9d3ace8b88"
      },
      "source": [
        "pool2d(X, (2, 2), 'avg')"
      ],
      "execution_count": 34,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "\n",
              "[[2. 3.]\n",
              " [5. 6.]]\n",
              "<NDArray 2x2 @cpu(0)>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 34
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "og2edxjbr8b5",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 122
        },
        "outputId": "462b6549-5096-465d-f347-01411000a744"
      },
      "source": [
        "X = nd.arange(16).reshape((1, 1, 4, 4))\n",
        "X"
      ],
      "execution_count": 35,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "\n",
              "[[[[ 0.  1.  2.  3.]\n",
              "   [ 4.  5.  6.  7.]\n",
              "   [ 8.  9. 10. 11.]\n",
              "   [12. 13. 14. 15.]]]]\n",
              "<NDArray 1x1x4x4 @cpu(0)>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 35
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "XOex1X4_sIpR",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 70
        },
        "outputId": "74a989c9-0a2b-4f37-aacf-867b2bbbdabe"
      },
      "source": [
        "pool2d = nn.MaxPool2D(3)\n",
        "pool2d(X)"
      ],
      "execution_count": 36,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "\n",
              "[[[[10.]]]]\n",
              "<NDArray 1x1x1x1 @cpu(0)>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 36
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "V6KBPNJMsYGr",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 87
        },
        "outputId": "49a84316-5fc6-4600-87a9-b00f1086208f"
      },
      "source": [
        "pool2d = nn.MaxPool2D(3, padding=1, strides=2)\n",
        "pool2d(X)"
      ],
      "execution_count": 37,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "\n",
              "[[[[ 5.  7.]\n",
              "   [13. 15.]]]]\n",
              "<NDArray 1x1x2x2 @cpu(0)>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 37
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6l9kanGoshR9",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 105
        },
        "outputId": "e6865fb2-547d-44e4-8936-6e6bf8196a78"
      },
      "source": [
        "pool2d = nn.MaxPool2D((2, 3), padding=(1, 2), strides=(2, 3))\n",
        "pool2d(X)"
      ],
      "execution_count": 38,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "\n",
              "[[[[ 0.  3.]\n",
              "   [ 8. 11.]\n",
              "   [12. 15.]]]]\n",
              "<NDArray 1x1x3x2 @cpu(0)>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 38
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "W5pEHoyOsx2h",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 210
        },
        "outputId": "8080ad47-48e8-4394-ae61-2af04bf17894"
      },
      "source": [
        "X = nd.concat(X, X + 1, dim=1)\n",
        "X "
      ],
      "execution_count": 39,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "\n",
              "[[[[ 0.  1.  2.  3.]\n",
              "   [ 4.  5.  6.  7.]\n",
              "   [ 8.  9. 10. 11.]\n",
              "   [12. 13. 14. 15.]]\n",
              "\n",
              "  [[ 1.  2.  3.  4.]\n",
              "   [ 5.  6.  7.  8.]\n",
              "   [ 9. 10. 11. 12.]\n",
              "   [13. 14. 15. 16.]]]]\n",
              "<NDArray 1x2x4x4 @cpu(0)>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 39
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "WJ5pULJAtIld",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 140
        },
        "outputId": "8c46d147-bc5a-4729-8ce7-deeabd590ecb"
      },
      "source": [
        "pool2d = nn.MaxPool2D(3, padding=1, strides=2)\n",
        "pool2d(X)"
      ],
      "execution_count": 42,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "\n",
              "[[[[ 5.  7.]\n",
              "   [13. 15.]]\n",
              "\n",
              "  [[ 6.  8.]\n",
              "   [14. 16.]]]]\n",
              "<NDArray 1x2x2x2 @cpu(0)>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 42
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "K69HpgadtQ0S",
        "colab_type": "text"
      },
      "source": [
        "## 5.5 LeNet"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GSG0WhsACSET",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import d2lzh as d2l \n",
        "import mxnet as mx \n",
        "from mxnet import autograd, gluon, init, nd \n",
        "from mxnet.gluon import loss as gloss, nn \n",
        "import time \n",
        "\n",
        "net = nn.Sequential()\n",
        "net.add(nn.Conv2D(channels=6, kernel_size=5, activation='sigmoid'), \n",
        "        nn.MaxPool2D(pool_size=2, strides=2), \n",
        "        nn.Conv2D(channels=16, kernel_size=5, activation='sigmoid'), \n",
        "        nn.MaxPool2D(pool_size=2, strides=2), \n",
        "        nn.Dense(120, activation='sigmoid'), \n",
        "        nn.Dense(84, activation='sigmoid'), \n",
        "        nn.Dense(10))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MOWilGDsCiRY",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 140
        },
        "outputId": "0fd812b7-ca14-404c-ce26-9d8d87a605d0"
      },
      "source": [
        "X = nd.random.uniform(shape=(1, 1, 28, 28))\n",
        "net.initialize()\n",
        "for layer in net:\n",
        "    X = layer(X)\n",
        "    print(layer.name, 'output shape:\\t', X.shape)"
      ],
      "execution_count": 45,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "conv6 output shape:\t (1, 6, 24, 24)\n",
            "pool6 output shape:\t (1, 6, 12, 12)\n",
            "conv7 output shape:\t (1, 16, 8, 8)\n",
            "pool7 output shape:\t (1, 16, 4, 4)\n",
            "dense0 output shape:\t (1, 120)\n",
            "dense1 output shape:\t (1, 84)\n",
            "dense2 output shape:\t (1, 10)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "QfJaYfEFEAI2",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 107
        },
        "outputId": "29b79ee5-8eba-4b6e-f253-38b9ea82a3f2"
      },
      "source": [
        "batch_size = 256 \n",
        "train_iter, test_iter = d2l.load_data_fashion_mnist(batch_size=batch_size)"
      ],
      "execution_count": 46,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Downloading /root/.mxnet/datasets/fashion-mnist/train-images-idx3-ubyte.gz from https://apache-mxnet.s3-accelerate.dualstack.amazonaws.com/gluon/dataset/fashion-mnist/train-images-idx3-ubyte.gz...\n",
            "Downloading /root/.mxnet/datasets/fashion-mnist/train-labels-idx1-ubyte.gz from https://apache-mxnet.s3-accelerate.dualstack.amazonaws.com/gluon/dataset/fashion-mnist/train-labels-idx1-ubyte.gz...\n",
            "Downloading /root/.mxnet/datasets/fashion-mnist/t10k-images-idx3-ubyte.gz from https://apache-mxnet.s3-accelerate.dualstack.amazonaws.com/gluon/dataset/fashion-mnist/t10k-images-idx3-ubyte.gz...\n",
            "Downloading /root/.mxnet/datasets/fashion-mnist/t10k-labels-idx1-ubyte.gz from https://apache-mxnet.s3-accelerate.dualstack.amazonaws.com/gluon/dataset/fashion-mnist/t10k-labels-idx1-ubyte.gz...\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "G-jneol2EYio",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "bc6742cd-bb72-4532-b85c-481e177c6a10"
      },
      "source": [
        "def try_gpu():\n",
        "    try:\n",
        "        ctx = mx.gpu()\n",
        "        _ = nd.zeros((1, ), ctx=ctx)\n",
        "    except mx.base.MXNetError:\n",
        "        ctx = mx.cpu()\n",
        "    return ctx \n",
        "\n",
        "ctx = try_gpu()\n",
        "ctx"
      ],
      "execution_count": 48,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "cpu(0)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 48
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nXqIfaJyEx7v",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def evaluate_accuracy(data_iter, net, ctx):\n",
        "    acc_sum, n = nd.array([0], ctx=ctx), 0\n",
        "    for X, y in data_iter:\n",
        "        # 如果ctx代表GPU及相应的显存，将数据复制到显存上\n",
        "        X, y = X.as_in_context(ctx), y.as_in_context(ctx).astype('float32')\n",
        "        acc_sum += (net(X).argmax(axis=1) == y).sum()\n",
        "        n += y.size\n",
        "    return acc_sum.asscalar() / n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "V3rfU33HFY9v",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def train_ch5(net, train_iter, test_iter, batch_size, trainer, ctx,\n",
        "              num_epochs):\n",
        "    print('training on', ctx)\n",
        "    loss = gloss.SoftmaxCrossEntropyLoss()\n",
        "    for epoch in range(num_epochs):\n",
        "        train_l_sum, train_acc_sum, n, start = 0.0, 0.0, 0, time.time()\n",
        "        for X, y in train_iter:\n",
        "            X, y = X.as_in_context(ctx), y.as_in_context(ctx)\n",
        "            with autograd.record():\n",
        "                y_hat = net(X)\n",
        "                l = loss(y_hat, y).sum()\n",
        "            l.backward()\n",
        "            trainer.step(batch_size)\n",
        "            y = y.astype('float32')\n",
        "            train_l_sum += l.asscalar()\n",
        "            train_acc_sum += (y_hat.argmax(axis=1) == y).sum().asscalar()\n",
        "            n += y.size\n",
        "        test_acc = evaluate_accuracy(test_iter, net, ctx)\n",
        "        print('epoch %d, loss %.4f, train acc %.3f, test acc %.3f, '\n",
        "              'time %.1f sec'\n",
        "              % (epoch + 1, train_l_sum / n, train_acc_sum / n, test_acc,\n",
        "                 time.time() - start))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "FL6DJ61DFel4",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 122
        },
        "outputId": "64186463-c270-4d6c-b09a-adfebc56da69"
      },
      "source": [
        "lr, num_epochs = 0.9, 5\n",
        "net.initialize(force_reinit=True, ctx=ctx, init=init.Xavier())\n",
        "trainer = gluon.Trainer(net.collect_params(), 'sgd', {'learning_rate': lr})\n",
        "train_ch5(net, train_iter, test_iter, batch_size, trainer, ctx, num_epochs)"
      ],
      "execution_count": 52,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "training on cpu(0)\n",
            "epoch 1, loss 2.3203, train acc 0.101, test acc 0.100, time 24.0 sec\n",
            "epoch 2, loss 1.9384, train acc 0.260, test acc 0.559, time 23.9 sec\n",
            "epoch 3, loss 0.9830, train acc 0.608, test acc 0.663, time 23.9 sec\n",
            "epoch 4, loss 0.7447, train acc 0.706, test acc 0.745, time 23.8 sec\n",
            "epoch 5, loss 0.6461, train acc 0.743, test acc 0.758, time 24.2 sec\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "H2aAkHYjFjGa",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 105
        },
        "outputId": "641f693d-bf43-4fdc-8dde-cdb06e718f80"
      },
      "source": [
        "!sudo lsb_release -a"
      ],
      "execution_count": 53,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "No LSB modules are available.\n",
            "Distributor ID:\tUbuntu\n",
            "Description:\tUbuntu 18.04.3 LTS\n",
            "Release:\t18.04\n",
            "Codename:\tbionic\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JFEt84cLG8jh",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "!apt-get update && sudo apt-get install -y build-essential git libgfortran3\n",
        "!wget https://developer.nvidia.com/compute/cuda/10.0/Prod/local_installers/cuda-repo-ubuntu1804-10-0-local-10.0.130-410.48_1.0-1_amd64\n",
        "!sudo dpkg -i cuda-repo-ubuntu1804-10-0-local-10.0.130-410.48_1.0-1_amd64.deb\n",
        "!sudo apt-key add /var/cuda-repo-<version>/7fa2af80.pub\n",
        "!sudo apt-get update\n",
        "!sudo apt-get install cuda"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "cxGaT0ikHntJ",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "3d879859-8ae8-435b-87e8-c741ebc86129"
      },
      "source": [
        "!cat /usr/local/cuda/version.txt"
      ],
      "execution_count": 55,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "CUDA Version 10.1.243\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "u3L7WocsI5wU",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "!pip install mxnet-cu100\n",
        "import mxnet as mx"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "UCAiCXh9I_DZ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}